package com.daisuxia.web.spider;

import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import net.sf.json.JSONArray;
import net.sf.json.JSONObject;

import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.daisuxia.constant.Constant;
import com.daisuxia.web.cache.CacheManager;
import com.daisuxia.web.cache.CacheTimerHandler;
import com.daisuxia.web.pojo.Content;

/**
 * 根据解析规则进行解析内容并返回
 * 
 * @author simalin
 * 
 */
public class Spider {

	private static Logger loger = Logger.getLogger(Spider.class);

	private static final Spider instance = new Spider();

	public static Spider getInstance() {
		return instance;
	}

	/**
	 * 根据关键字：infoKeyword解析html
	 * 
	 * @param infoValue
	 *            : 请求参数
	 * @param stype
	 *            : 1：天猫 2：京东
	 * @param pType
	 *            : 0：详情 1：列表
	 * @return
	 */
	public List<HashMap<String, Object>> parse(String infoValue,
			String parseType) {
		long startDate = new Date().getTime();
		loger.info("---->parse-html-start--:" + startDate);
		/*
		 * 获取列表以及详情页的缓存数据 CacheTimerHandler resultCache = null;
		 * if(parseType.equals(Constant.STATUS_TYPE_ONE)){//列表页 resultCache =
		 * CacheManager.getInstance().getCacheTimerHandlerByList(); }else
		 * if(parseType.equals(Constant.STATUS_TYPE_ZREO)){//详情页 resultCache =
		 * CacheManager.getInstance().getCacheTimerHandlerByItem(); }
		 */

		Map<String, Object> map = new HashMap<String, Object>();

		CacheTimerHandler cache = CacheManager.getInstance()
				.getCacheTimerHandler();// 获取缓存中现有的规则
		List<SchemeRuleDto> schemeRuleList = null;
		try {
			if (null == cache
					|| null == cache.getCache(Constant.SCHEME).getCacheKey(
							Constant.SCHEME)) {
				// 当缓存中为空时，获取、更新网页抓取方案及规则
				ParseSchemeService.getInstance().loadParseSchemes();
			}
		} catch (Exception e) {
			ParseSchemeService.getInstance().loadParseSchemes();
		}

		schemeRuleList = ParseSchemeService.getInstance()
				.getAllSchmesRuleList();// 获取规则列表-->list
		if (null == schemeRuleList) {
			loger.info("-!-!-!-get-scheme-fail-!-!-!-");
			return null;
		}

		SchemeRuleDto schemeRuleDto = null;// 存放规则中主属性
		List<HashMap<String, Object>> AllListMap = new ArrayList<HashMap<String, Object>>();// 存放解析结果集
		List<HashMap<String, Object>> listMap = new ArrayList<HashMap<String, Object>>();// 存放解析结果集

		String siteType = "";// 默认1：天猫 2：京东

		if (StringUtils.isBlank(siteType)) {
			siteType = "1";// 默认1：天猫 2：京东
		}
		String url = "";
		Document doc = null;

		if (parseType.equals(Constant.SCHEME_TYPE_LIST)) {// 列表
			// 获取列表输入参数
			Map<String, Object> paramMap = new HashMap<String, Object>();
			List<String> paramList = new ArrayList<String>();
			paramMap = RequestUtil.loadInfoValue(infoValue, parseType);
			paramList = (List<String>) paramMap.get(Constant.SOURCES_MAP_KEY);

			/*
			 * 获取列表缓存,如果缓存中存在此keyword的数据，则直接返回 resultCache =
			 * CacheManager.getInstance
			 * ().getCacheTimerHandlerByList();//获取列表缓存,如果缓存中存在此keyword的数据，则直接返回
			 * if(null!= resultCache && null !=
			 * resultCache.getCache("LIST").getCacheKey
			 * (String.valueOf(paramMap.get(Constant.KEYWORD_MAP_KEY)))){
			 * 
			 * return (List<HashMap<String, Object>>)
			 * resultCache.getCache("LIST"
			 * ).getCacheKey(String.valueOf(paramMap.get
			 * (Constant.KEYWORD_MAP_KEY))); }
			 */

			for (String pUrl : paramList) {
				url = pUrl;
				loger.info("---url--+" + url);
				if (url.contains(Constant.WDTY_LIST)) {
					siteType = Constant.SCHEME_STATUS_ONE;
				} else if (url.contains(Constant.WDZJ_LIST)) {
					siteType = Constant.SCHEME_STATUS_TWO;
				}
				try {

					schemeRuleDto = getSchemeRuleDto(schemeRuleList, siteType,
							parseType);// 获取规则
					loger.info("选用规则SchemeRuleDto=" + schemeRuleDto);

					doc = ParseImpl.getDocmentByUrl(url);

					listMap = parseForList(doc, schemeRuleDto);// 列表结果集返回
				} catch (Exception e) {
					e.printStackTrace();
					loger.info("-!-!-!-get-html-fail-!-!-!-");
				}
				AllListMap.addAll(listMap);
			}

			// CacheEntity ce = new
			// CacheEntity(Constant.KEYWORD_MAP_KEY,AllListMap); //把数据放入缓存
			// resultCache.getCache("LIST").setCacheContext(ce);

		} else {// 获取详情页信息
			Map<String, Object> paramMap = new HashMap<String, Object>();
			List<String> paramList = new ArrayList<String>();
			paramMap = RequestUtil.loadInfoValue(infoValue, parseType);
			paramList = (List<String>) paramMap.get(Constant.SOURCES_MAP_KEY);
			if (null != paramMap) {
				url = String.valueOf(paramMap.get("url"));
			}
			if (url.contains(Constant.WDTY_LIST)) {//
				siteType = Constant.SCHEME_STATUS_ONE;
			} else if (url.contains(Constant.WDZJ_LIST)) {
				siteType = Constant.SCHEME_STATUS_TWO;
			}
			schemeRuleDto = getSchemeRuleDto(schemeRuleList, siteType,
					parseType);// 获取规则
			doc = ParseImpl.getDocmentByUrl(url);
			loger.info("doc==" + doc);
			AllListMap = parseForDetail(doc, schemeRuleDto, siteType);// 详情结果集返回

			// CacheEntity ce = new
			// CacheEntity(Constant.KEYWORD_MAP_KEY,AllListMap);//把数据放入缓存
			// resultCache.getCache("LIST").setCacheContext(ce);
		}

		loger.info("获取属性siteType=" + siteType + " parseType=" + parseType);

		long endDate = new Date().getTime();
		loger.info("---->parse-html-end--:" + endDate + "  共计用时:"
				+ (endDate - startDate) + " 获取size=" + map.size());

		return AllListMap;
	}

	/**
	 * 详情页解析
	 * 
	 * @param doc
	 * @param schemeRuleDto
	 * @return
	 */
	public List<HashMap<String, Object>> parseForDetail(Document doc,
			SchemeRuleDto schemeRuleDto, String siteType) {
		int num = 1;
		List<HashMap<String, Object>> listMap = new ArrayList<HashMap<String, Object>>();
		List<RulesDto> rulesList = new ArrayList<RulesDto>();
		HashMap<String, Object> valMap = null;
		RulesDto rulesDto = new RulesDto();
		rulesList = schemeRuleDto.getRulesList();
		for (int i = 0; i < rulesList.size(); i++) {
			valMap = new HashMap<String, Object>();
			rulesDto = rulesList.get(i);

			/* 根据不同的参数属性解析doc */
			String obj = "";
			loger.info("rulesDto.getParamType=" + rulesDto.getParamType());
			if (StringUtils.isBlank(rulesDto.getParamType())) {
				Element el = ParseImpl.getElement(doc, rulesDto.getDocSelect(),
						rulesDto.getDocInfoKey(), rulesDto.getDocInfoValue());
				obj = ParseImpl.getVaule(el, rulesDto.getSelect(), rulesDto
						.getInfoKey(), rulesDto.getInfoValue());
			} else if ("list".equals(rulesDto.getParamType())) {
				obj = ParseImpl.getVaule(ParseImpl.getElement(doc, rulesDto
						.getDocSelect(), rulesDto.getDocInfoKey(), rulesDto
						.getDocInfoValue()), rulesDto.getSelect(), rulesDto
						.getInfoKey(), rulesDto.getInfoValue());
			} else if ("script".equals(rulesDto.getParamType())) {
				if (siteType.equals(Constant.SCHEME_STATUS_ONE)) {

					obj = parseForDetailByScript(doc, rulesDto);
				} else if (siteType.equals(Constant.SCHEME_STATUS_TWO)) {
					obj = parseForDetailByScripts(doc, rulesDto);
				}

			} else if (rulesDto.getParamType().startsWith("url")) {

				obj = parseForDetailByUrl(doc, rulesDto);
			}
			loger.info("详情页第" + num + "元素" + rulesDto.getParam() + ":" + obj);
			// 获取价格
			// if("skudetail".equals(rulesDto.getParam())){
			// obj = PriceUtil.getPriceByDetail(obj, siteType);
			// }

			valMap.put(rulesDto.getParam(), obj);
			listMap.add(valMap);
		}

		loger.info("listMap=" + listMap.toString());
		return listMap;
	}

	/**
	 * 处理script类型 当需要处理的类型为script时，查询需要的script文件，从中获取需要的内容
	 * 
	 * @param doc
	 * @param rulesDto
	 * @return
	 */

	public String parseForDetailByScript(Document doc, RulesDto rulesDto) {
		String result = parseForDetailByScripts(doc, rulesDto);
		JSONObject jsonObj = JSONObject.fromObject((result));
		JSONObject jb = jsonObj.getJSONObject("valItemInfo");
		loger.info("js=" + jb.toString());
		return jb.getString("skuList");
	}

	/*
	 * public String parseForDetailByScript(Document doc, RulesDto rulesDto) {
	 * String docInfoValue = rulesDto.getDocInfoValue(); Elements eles =
	 * ParseImpl.getElementByTag(doc, rulesDto.getDocSelect()); String scriptUrl
	 * = ""; HashMap<String, Object> map = new HashMap<String, Object>(); for
	 * (Element e : eles) {// 获取js中页面子路径 String script = e.data().trim(); if
	 * (script.contains(docInfoValue)) { scriptUrl = script.substring(
	 * script.indexOf(docInfoValue), script.indexOf(rulesDto.getDocInfoKey(),
	 * script.indexOf(docInfoValue))); break; } }
	 * loger.info("--detail - js - url -->" + scriptUrl); String obj = ""; if
	 * (StringUtils.isNotBlank(scriptUrl)) { Document scriptDoc =
	 * ParseImpl.getDocmentByUrl("http:" + scriptUrl); Element element =
	 * scriptDoc.body();
	 * 
	 * obj = ParseImpl.getVaule(element, rulesDto.getSelect(),
	 * rulesDto.getInfoKey(), rulesDto.getInfoValue()); }
	 * 
	 * return obj; }
	 */
	/**
	 * 处理script，其中为完整的json字符串
	 * 
	 * @param doc
	 * @param rulesDto
	 * @return
	 */
	public String parseForDetailByScripts(Document doc, RulesDto rulesDto) {
		String docInfoValue = rulesDto.getDocInfoValue();
		String docSelect = rulesDto.getSelect();
		Elements eles = ParseImpl.getElementByTag(doc, rulesDto.getDocSelect());

		String scriptUrl = "";
		String scriptStr = "";

		HashMap<String, Object> map = new HashMap<String, Object>();
		for (Element e : eles) {// 获取js中页面子路径
			String script = e.data().trim();
			if (script.contains(docInfoValue)) {
				if (StringUtils.isNotBlank(docSelect)) {
					if (script.contains(docSelect)) {
						scriptStr = script.substring(script.indexOf(docSelect),
								script.length());
					}
				} else {
					scriptStr = script.substring(script.indexOf(docInfoValue),
							script.length());
				}

				loger.info("scriptStr=" + scriptStr);
				loger.info("rulesDto.getDocInfoKey()="
						+ rulesDto.getDocInfoKey());
				if (Constant.SCHEME_STATUS_ZERO
						.equals(rulesDto.getDocInfoKey())) {
					scriptUrl = scriptStr.substring(scriptStr.indexOf(rulesDto
							.getInfoKey()), scriptStr.indexOf(rulesDto
							.getInfoValue()) + 1);
				} else if (Constant.SCHEME_STATUS_ONE.equals(rulesDto
						.getDocInfoKey())) {
					scriptUrl = scriptStr.substring(scriptStr.indexOf(rulesDto
							.getInfoKey()) + 1, scriptStr.indexOf(rulesDto
							.getInfoValue()));
				}
				break;
			}
		}
		loger.info("--detail - js - url -->" + scriptUrl);
		return scriptUrl;
	}

	/**
	 * 通过url获取元素值 当获取的内容需要再次访问url时
	 * 
	 * @param doc
	 * @param rulesDto
	 * @return
	 */
	public String parseForDetailByUrl(Document doc, RulesDto rulesDto) {
		try {
			String[] invalue = rulesDto.getParamType().split(",");
			Element el = ParseImpl.getElement(doc, rulesDto.getDocSelect(),
					rulesDto.getDocInfoKey(), rulesDto.getDocInfoValue());
			String newVal = ParseImpl.getVaule(el, rulesDto.getSelect(),
					rulesDto.getInfoKey(), rulesDto.getInfoValue());
			String url = invalue[1].replace(invalue[2], newVal);
			JSONArray jsonl = JSONArray.fromObject(HttpUtil.postSends(url));

			String obj = jsonl.getJSONObject(0).getString("p");
			return obj;
		} catch (Exception e) {
			return null;
		}
	}

	/**
	 * 列表页解析
	 * 
	 * @param doc
	 * @param schemeRuleDto
	 * @return
	 */
	public List<HashMap<String, Object>> parseForList(Document doc,
			SchemeRuleDto schemeRuleDto) {
		int num = 1;
		List<HashMap<String, Object>> listMap = new ArrayList<HashMap<String, Object>>();
		List<RulesDto> rulesList = new ArrayList<RulesDto>();
		HashMap<String, Object> valMap = null;
		RulesDto rulesDto = null;

		Elements eles = ParseImpl.getElements(doc, schemeRuleDto.getSelect(),
				schemeRuleDto.getInfoKey(), schemeRuleDto.getInfoValue());

		for (Element ele : eles) {
			rulesList = schemeRuleDto.getRulesList();
			valMap = new HashMap();
			for (int j = 0; j < rulesList.size(); j++) {
				rulesDto = rulesList.get(j);

				String obj = ParseImpl.getVaule(ele, rulesDto.getSelect(),
						rulesDto.getInfoKey(), rulesDto.getInfoValue());

				loger.info("列表页第" + num + "元素" + rulesDto.getParam() + ":"
						+ obj);
				if (StringUtils.isNotBlank(obj)) {
					valMap.put(rulesDto.getParam(), obj);
				}
			}
			listMap.add(valMap);
			num++;
		}
		return listMap;
	}

	/**
	 * 获取存放规则中主属性
	 * 
	 * @param schemeRuleList
	 * @param siteType站点类型1
	 *            .网贷天眼，2网贷之家
	 * @param parseType转换类型1
	 *            .列表，2详情
	 * @return
	 */
	public SchemeRuleDto getSchemeRuleDto(List<SchemeRuleDto> schemeRuleList,
			String siteType, String parseType) {
		List<SchemeRuleDto> allSchemeRuleList = new ArrayList<SchemeRuleDto>();// 存放列表规则
		// :
		// 如果存在多个规则的情况下，当一个规则爬取失败时，可以更换为另一个规则或者随机个规则
		for (int i = 0; i < schemeRuleList.size(); i++) {
			SchemeRuleDto srd = schemeRuleList.get(i);
			loger.info("规则SchemeRuleDto=" + srd);
			if (srd.getStatus().equals(Constant.SCHEME_TYPE_LIST)
					&& srd.getSiteId().equals(siteType)
					&& srd.getType().equals(parseType)) {
				allSchemeRuleList.add(srd);
			}
		}
		return allSchemeRuleList.get(0);// 当有多个规则存在时,可根据具体设置的权重来随机更换规则：Random(allSchemeRuleList.size);

	}

	public static void main(String[] args) {
//		System.out.println("<p>sdfd<a href='www.baidu.com' class=\"11\">1231</a></p>".replaceAll("<a[^<]*>", ""));
		System.exit(0);
		ParseSchemeService.getInstance().loadParseSchemes();
		Document doc = ParseImpl.getDocmentByUrl(Constant.WDTY_LIST);
		List<SchemeRuleDto> list = ParseSchemeService.getInstance()
				.getAllSchmesRuleList();
		SchemeRuleDto schemeRuleDto = Spider.getInstance().getSchemeRuleDto(
				list, "1", "1");
		List<HashMap<String, Object>> list2 = Spider.getInstance()
				.parseForList(doc, schemeRuleDto);
		List<Content> contentsList = new ArrayList<Content>();
		for (HashMap<String, Object> hashMap : list2) {
			Content content = new Content();
			content
					.setContentTitle(String
							.valueOf(hashMap.get("contentTitle")));
			content.setContentSummary(String.valueOf(hashMap
					.get("contentSummary")));
			if (hashMap.containsKey("detailUrl")) {
				Document doc2 = ParseImpl.getDocmentByUrl(String
						.valueOf(hashMap.get("detailUrl")));
				schemeRuleDto = Spider.getInstance().getSchemeRuleDto(list,
						"1", "0");
				List<HashMap<String, Object>> list3 = Spider.getInstance()
						.parseForDetail(doc2, schemeRuleDto, "list");
				content.setContentTxt(String.valueOf(list3.get(0).get(
						"contentTxt")));
				System.out.println("before>>>>>>>>>" + content.getContentTxt());
				System.out.println("after>>>>>>>>>"
						+ content.getContentTxt().replaceAll(
								"<[^<]*>", ""));
				contentsList.add(content);
			}
		}
		System.exit(0);
	}
}
